<!DOCTYPE html>
<html  lang="zh-CN" >
    <head>
    <meta charset="UTF-8">
    <meta name="viewport" content="width=device-width, minimum-scale=1, initial-scale=1, maximum-scale=5, viewport-fit=cover">
    <title>Kudu架构设计解读 | 学为我用</title>
    <meta name="description" content="简介首先,kudu是一个列式存储的数据库，他是hadoop生态系统的一员，支持可横向扩展和高可用。kudu可以用于以下几种场景。  OLAP快速处理 结合mapReduce,Spark和其他的Hadoop组件做开发 可以和impala非常完美的集成使用，使之可以更好更灵活的访问HDFS上的parquet文件 强大且灵活的一致性保障机制，可以基于每次请求做一次性保证，包含序列访问的一致性保证 顺序和">
<meta property="og:type" content="article">
<meta property="og:title" content="Kudu架构设计解读">
<meta property="og:url" content="https://123zhangzhenbin.gitee.io/zzbomb/2020/10/12/202010121559-kudu-arch/index.html">
<meta property="og:site_name" content="学为我用">
<meta property="og:description" content="简介首先,kudu是一个列式存储的数据库，他是hadoop生态系统的一员，支持可横向扩展和高可用。kudu可以用于以下几种场景。  OLAP快速处理 结合mapReduce,Spark和其他的Hadoop组件做开发 可以和impala非常完美的集成使用，使之可以更好更灵活的访问HDFS上的parquet文件 强大且灵活的一致性保障机制，可以基于每次请求做一次性保证，包含序列访问的一致性保证 顺序和">
<meta property="og:locale" content="zh_CN">
<meta property="og:image" content="https://123zhangzhenbin.gitee.io/zzbomb/2020/10/12/202010121559-kudu-arch/s1.png">
<meta property="article:published_time" content="2020-10-12T07:59:31.000Z">
<meta property="article:modified_time" content="2020-10-12T09:28:56.587Z">
<meta property="article:author" content="章瑱彬">
<meta property="article:tag" content="大数据">
<meta name="twitter:card" content="summary">
<meta name="twitter:image" content="https://123zhangzhenbin.gitee.io/zzbomb/2020/10/12/202010121559-kudu-arch/s1.png">

    
    <link rel="icon" href="/zzbomb/images/favicon_128.ico" type="image/x-icon">

    
<link rel="stylesheet" href="/zzbomb/css/common.min.css">



    
    
    
    
        <link href="//cdn.jsdelivr.net/npm/lightgallery.js@1.1.3/dist/css/lightgallery.min.css" rel="stylesheet">
    
    
    
<link rel="stylesheet" href="/zzbomb/css/iconfont.min.css">

    
<meta name="generator" content="Hexo 4.2.1"></head>

    <body>
        <header class="header header-fixture">
    <div class="profile-search-wrap flex sm:block">
        
        
        <div class="profile sm:text-center md:px-1 lg:px-3 sm:pb-4 sm:pt-6">
            <a id="avatar" role="link" href="https://gitee.com/123zhangzhenbin" class="inline-block lg:w-16 lg:h-16 w-8 h-8 m-2" target="_blank" rel="noopener" rel="noreferrer" >
                <img src="/zzbomb/images/g.jpg" class="rounded-full" alt="avatar">
            </a>
            <h2 id="name" class="hidden lg:block">ZZB</h2>
            <h3 id="title" class="hidden xl:block">Java，大数据，人工智能，区块链</h3>
            
            <small id="location" class="hidden lg:block">
                <i class="iconfont icon-map-icon"></i>
                DongGuan, China
            </small>
            
        </div>
        
        
<div class="search flex-1 flex lg:inline-block sm:hidden lg:px-4 lg:mt-2 lg:mb-4 lg:w-full">
    <form id="search-form" class="my-auto flex-1 lg:border lg:border-solid lg:border-gray-200">
        <div class="input-group table bg-gray-100 lg:bg-white w-full">
            <input id="search-input" type="text" placeholder="搜索" class="inline-block w-full bg-gray-100 lg:bg-white">
            <span class="table-cell">
                <button name="search tigger button" disabled>
                    <i class="iconfont icon-search m-2"></i>
                </button>
            </span>
        </div>
    </form>
        
<script id="search-teamplate" type="text/html" data-path="/zzbomb/content.json">
    <div>
        <div class="search-header bg-gray-400">
            <input id="actual-search-input" model="keyword" ref="input" class="inline-block w-full h-10 px-2 py-1" placeholder="搜索" type="text">
        </div>
        <div class="search-result bg-gray-200">
            {{#each searchPosts}}
            <a href="/{{ path }}" class="result-item block px-2 pb-3 mb-1 pt-1 hover:bg-indigo-100">
                <i class="iconfont icon-file"></i>
                <h1 class="result-title inline font-medium text-lg">{{ title }}</h1>
                <p class="result-content text-gray-600 text-sm">{{{ text }}}</p>
            </a>
            {{/each}}
        </div>
    </div>
</script>

</div>


        <button name="menu toogle button" id="menu-toggle-btn" class="block sm:hidden p-3" role="button" aria-expanded="false">
            <i class="iconfont icon-hamburger"></i>
        </button>
    </div>
    <nav id="menu-nav" class="hidden sm:flex flex-col">
        
        
            <div class="menu-item menu-home" role="menuitem">
                <a href="/zzbomb/.">
                    <i class="iconfont icon-home" aria-hidden="true"></i>
                    <span class="menu-title">首页</span>
                </a>
            </div>
        
        
            <div class="menu-item menu-archives" role="menuitem">
                <a href="/zzbomb/archives">
                    <i class="iconfont icon-archive" aria-hidden="true"></i>
                    <span class="menu-title">归档</span>
                </a>
            </div>
        
        
            <div class="menu-item menu-categories" role="menuitem">
                <a href="/zzbomb/categories">
                    <i class="iconfont icon-folder" aria-hidden="true"></i>
                    <span class="menu-title">分类</span>
                </a>
            </div>
        
        
            <div class="menu-item menu-tags" role="menuitem">
                <a href="/zzbomb/tags">
                    <i class="iconfont icon-tag" aria-hidden="true"></i>
                    <span class="menu-title">标签</span>
                </a>
            </div>
        
        
            <div class="menu-item menu-repository" role="menuitem">
                <a href="/zzbomb/repository">
                    <i class="iconfont icon-project" aria-hidden="true"></i>
                    <span class="menu-title">项目</span>
                </a>
            </div>
        
        
            <div class="menu-item menu-links" role="menuitem">
                <a href="/zzbomb/links">
                    <i class="iconfont icon-friend" aria-hidden="true"></i>
                    <span class="menu-title">友链</span>
                </a>
            </div>
        
        
            <div class="menu-item menu-about" role="menuitem">
                <a href="/zzbomb/about">
                    <i class="iconfont icon-cup" aria-hidden="true"></i>
                    <span class="menu-title">关于</span>
                </a>
            </div>
        
        
<div class="social-links flex sm:flex-col lg:hidden mt-5">
    
        <span class="social-item text-center">
            <a href="https://gitee.com/123zhangzhenbin" target="_blank" rel="noopener">
                <i class="iconfont social-icon icon-github"></i>
                <span class="menu-title hidden lg:inline">menu.github</span>
            </a>
        </span>
    
</div>


    </nav>
</header>

        <section class="main-section">
            
    <main class="flex-1 px-4 py-12 md:px-5 lg:px-8 lg:py-4 relative min-h-screen">
    

    <article class="content article article-archives article-type-list" itemscope="">
        <header class="article-header">
            
    
        <h1 class="article-title text-lg" itemprop="name">
            Kudu架构设计解读
        </h1>
    



            <p class="article-meta mb-3 text-xs">
                <span class="article-date">
    <i class="iconfont icon-calendar-check"></i>
	<a href="/zzbomb/2020/10/12/202010121559-kudu-arch/" class="article-date">
	  <time datetime="2020-10-12T07:59:31.000Z" itemprop="datePublished">10月 12</time>
	</a>
</span>

                

                
    <span class="article-tags">
    <i class="iconfont icon-tag"></i>
    <a class="article-tag-link" href="/zzbomb/tags/%E5%A4%A7%E6%95%B0%E6%8D%AE/" rel="tag">大数据</a>
  </span>


                <span class="_partial/post-comment"><i class="icon icon-comment"></i>
                    <a href="/zzbomb/2020/10/12/202010121559-kudu-arch/#comments" class="article-comment-link">
                        评论
                    </a>
                </span>
                

            </p>
        </header>
        <div class="marked-body article-body">
            <h2 id="简介"><a href="#简介" class="headerlink" title="简介"></a>简介</h2><p>首先,kudu是一个列式存储的数据库，他是hadoop生态系统的一员，支持可横向扩展和高可用。kudu可以用于以下几种场景。</p>
<ol>
<li>OLAP快速处理</li>
<li>结合mapReduce,Spark和其他的Hadoop组件做开发</li>
<li>可以和impala非常完美的集成使用，使之可以更好更灵活的访问HDFS上的parquet文件</li>
<li>强大且灵活的一致性保障机制，可以基于每次请求做一次性保证，包含序列访问的一致性保证</li>
<li>顺序和随机处理均能提供较好的性能</li>
<li>方便管理和监控</li>
<li>高可用，TabletServers和Master使用Raft一致性算法，保障了过半即可用。</li>
<li>结构化数据模型</li>
</ol>
<p>在以下案例中，用kudu可以完美解决：</p>
<ol>
<li>交互式增量计算，立刻返回给用户。</li>
<li>时序数据处理。</li>
<li>基于增量或全量的实时决策系统。</li>
</ol>
<h2 id="专业术语"><a href="#专业术语" class="headerlink" title="专业术语"></a>专业术语</h2><h3 id="Columnar-Data-Store"><a href="#Columnar-Data-Store" class="headerlink" title="Columnar Data Store"></a>Columnar Data Store</h3><p>列式存储，kudu是一个列式存储的，且强类型的数据库。</p>
<h3 id="Read-Efficiency"><a href="#Read-Efficiency" class="headerlink" title="Read Efficiency"></a>Read Efficiency</h3><p>高效读取。因为kudu是列式的，相较于行式数据库需要读取一整行来获取字段数据来说，kudu的列式读取更加高效</p>
<h3 id="Data-Compression"><a href="#Data-Compression" class="headerlink" title="Data Compression"></a>Data Compression</h3><p>数据压缩。支持LZ4，Snappy或zlib，追求查询效率推荐使用LZ4，追求压缩率推荐使用zlib。</p>
<h3 id="Tablet"><a href="#Tablet" class="headerlink" title="Tablet"></a>Tablet</h3><p>tablet是一端连续存储的数据，类似于其他存储引擎中的分区概念。tablet的副本分散在各个tabletServer中，主节点保证写入一致性。任何的副本都可以读取数据。但写入是靠主节点同步写入的。</p>
<h3 id="TabletServer"><a href="#TabletServer" class="headerlink" title="TabletServer"></a>TabletServer</h3><p>tableServer用来存储tablet。在一个tablet中，其中一个tabletServer用来做主节点，其他的tabletServer用来做从节点。所有的节点都可以读，但是只有主节点可以写入数据。主节点通过raft一致性算法来保证一致性。一个tabletServer可以装载多个tablet，一个tablet也可以被多个tableServer装载。</p>
<h3 id="Master"><a href="#Master" class="headerlink" title="Master"></a>Master</h3><p>master服务用来监控tablet,tabletserver,catalogtable和存储一些其他的元数据。<br>master的高可用是主备机制，使用Raft一致性算法。Master同时负责对请求进行分发，到各个tabletServer。<br>所有master的数据都存在tablet中，并且在每一个tabletServer中都有副本。</p>
<h3 id="Raft-Consensus-Algorithm"><a href="#Raft-Consensus-Algorithm" class="headerlink" title="Raft Consensus Algorithm"></a>Raft Consensus Algorithm</h3><p>一种基于投票的一致性算法，类似于Paxos。</p>
<h3 id="Catalog-Table"><a href="#Catalog-Table" class="headerlink" title="Catalog Table"></a>Catalog Table</h3><p>元数据。一般包含以下两种：</p>
<ul>
<li>tables<br>  表的schema,locations和states</li>
<li>tablets<br>  当前tablets列表及其元数据</li>
</ul>
<h3 id="Logical-Replication"><a href="#Logical-Replication" class="headerlink" title="Logical Replication"></a>Logical Replication</h3><p>逻辑副本</p>
<h2 id="架构图"><a href="#架构图" class="headerlink" title="架构图"></a>架构图</h2><p><img src="/zzbomb/2020/10/12/202010121559-kudu-arch/s1.png" alt="s1"></p>

        </div>
        
<blockquote class="copyright">
    <p><strong>本文链接 : </strong><a class="permalink" href="https://123zhangzhenbin.gitee.io/zzbomb/2020/10/12/202010121559-kudu-arch/">https://123zhangzhenbin.gitee.io/zzbomb/2020/10/12/202010121559-kudu-arch/</a></p>
    <p><strong>This article is available under <a href="https://creativecommons.org/licenses/by-nc-sa/4.0/" target="_blank" rel="noopener noreferrer">Attribution-NonCommercial-ShareAlike 4.0 International (CC BY-NC-SA 4.0)</a> License</strong></p>
</blockquote>


    </article>
    
    <section id="comments">
        
    </section>


    

</main>


<aside style="" id="sidebar" class="aside aside-fixture">
    <div class="toc-sidebar">
        <nav id="toc" class="article-toc">
            <h3 class="toc-title">文章目录</h3>
            <ol class="toc"><li class="toc-item toc-level-2"><a class="toc-link" href="#简介"><span class="toc-number">1.</span> <span class="toc-text">简介</span></a></li><li class="toc-item toc-level-2"><a class="toc-link" href="#专业术语"><span class="toc-number">2.</span> <span class="toc-text">专业术语</span></a><ol class="toc-child"><li class="toc-item toc-level-3"><a class="toc-link" href="#Columnar-Data-Store"><span class="toc-number">2.1.</span> <span class="toc-text">Columnar Data Store</span></a></li><li class="toc-item toc-level-3"><a class="toc-link" href="#Read-Efficiency"><span class="toc-number">2.2.</span> <span class="toc-text">Read Efficiency</span></a></li><li class="toc-item toc-level-3"><a class="toc-link" href="#Data-Compression"><span class="toc-number">2.3.</span> <span class="toc-text">Data Compression</span></a></li><li class="toc-item toc-level-3"><a class="toc-link" href="#Tablet"><span class="toc-number">2.4.</span> <span class="toc-text">Tablet</span></a></li><li class="toc-item toc-level-3"><a class="toc-link" href="#TabletServer"><span class="toc-number">2.5.</span> <span class="toc-text">TabletServer</span></a></li><li class="toc-item toc-level-3"><a class="toc-link" href="#Master"><span class="toc-number">2.6.</span> <span class="toc-text">Master</span></a></li><li class="toc-item toc-level-3"><a class="toc-link" href="#Raft-Consensus-Algorithm"><span class="toc-number">2.7.</span> <span class="toc-text">Raft Consensus Algorithm</span></a></li><li class="toc-item toc-level-3"><a class="toc-link" href="#Catalog-Table"><span class="toc-number">2.8.</span> <span class="toc-text">Catalog Table</span></a></li><li class="toc-item toc-level-3"><a class="toc-link" href="#Logical-Replication"><span class="toc-number">2.9.</span> <span class="toc-text">Logical Replication</span></a></li></ol></li><li class="toc-item toc-level-2"><a class="toc-link" href="#架构图"><span class="toc-number">3.</span> <span class="toc-text">架构图</span></a></li></ol>
        </nav>
    </div>
</aside>





        </section>
        <footer class="hidden lg:block fixed bottom-0 left-0 sm:w-1/12 lg:w-1/6 bg-gray-100 z-40">
    
    <div class="footer-social-links">
        
            <a href="https://gitee.com/123zhangzhenbin" target="_blank" rel="noopener">
                <i class="iconfont icon-github"></i>
            </a>
        
    </div>
    
    <p style="display: block;text-align: center;font-size: 13px;"><a href="https://beian.miit.gov.cn/" target="_blank">粤ICP备16013026号</a></p>
    
        <p class="footer-custom"><a href="https://beian.miit.gov.cn/" target="_blank">粤ICP备16013026号</a></p>
        <p class="theme-brand">Theme by <a href="https://github.com/fengkx/hexo-theme-purer" target="_blank" rel="nofollow noopener noreferrer noopener">hexo-theme-purer</a></p>

    
</footer>

        <div id="mask" class="hidden mask fixed inset-0 bg-gray-900 opacity-75 z-40"></div>
        <div id="search-view-container" class="hidden shadow-xl"></div>
        
<script src="/zzbomb/js/dom-event.min.js"></script>

<script src="//cdn.jsdelivr.net/npm/yox@1.0.0-alpha.121/dist/standard/prod/yox.min.js"></script>


<script src="/zzbomb/js/search.min.js"></script>



    <script src="//cdn.jsdelivr.net/npm/lightgallery.js@1.1.3/dist/js/lightgallery.min.js"></script>
    
<script src="/zzbomb/js/light-gallery.min.js"></script>





    </body>
</html>
